package main;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.PrintStream;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Enumeration;
import java.util.HashSet;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;

import opennlp.tools.sentdetect.SentenceDetectorME;
import opennlp.tools.sentdetect.SentenceModel;
import opennlp.tools.util.Span;

import org.mitre.jawb.io.SgmlDocument;
import org.mitre.jawb.io.SgmlElement;

import edu.stanford.nlp.ling.CoreLabel;
import edu.stanford.nlp.ling.CoreAnnotations.PartOfSpeechAnnotation;
import edu.stanford.nlp.ling.CoreAnnotations.TextAnnotation;
import edu.stanford.nlp.ling.CoreAnnotations.TokensAnnotation;
import edu.stanford.nlp.pipeline.Annotation;
import edu.stanford.nlp.pipeline.StanfordCoreNLP;
import edu.washington.cs.knowitall.extractor.mapper.ReVerbRelationDictionaryFilter;
import edu.washington.cs.knowitall.nlp.ChunkedSentence;
import edu.washington.cs.knowitall.nlp.OpenNlpChunkedSentenceParser;
import edu.washington.cs.knowitall.nlp.OpenNlpSentenceChunker;
import edu.washington.cs.knowitall.util.DefaultObjects;

public class Chunking {

	private static SentenceDetectorME sentenceDetector;
	
	private static final String NATURAL_DISASTER = "NaturalDisaster";

	private static Hashtable<String, Integer> midAccumTextCount;

	private static ArrayList<String> midAccumTexts;

	private static Hashtable<String, Integer> midTextCount;

	private static ArrayList<String> midTexts;
	
	private static Map<String, Hashtable<String,Set<String>>> argumentsCombination = new Hashtable<String, Hashtable<String,Set<String>>>();
	
	private static Hashtable<String, Set<String>> allArgument = new Hashtable<String, Set<String>>();
	
	public static String[] Datasets = {"AFFILIATE-PARTNER","ASSOCIATE","AT","BASED-IN","CITIZEN-OF","CLIENT","FOUNDER","GENERAL-STAFF","GRANDPARENT",
			"LOCATED", "MANAGEMENT","MEMBER","NEAR","OTHER","OTHER-PERSONAL","OTHER-PROFESSIONAL","OTHER-RELATIVE","OWNER","PARENT",
			"PART","PART-OF","RELATIVE-LOCATION","RESIDENCE","ROLE","SIBLING","SOC","SPOUSE","SUBSIDIARY","NaturalDisaster"};
	
	
	/**
	 * @param args
	 * @throws IOException 
	 */
	public static void main(String[] args) throws IOException {
		
		sentenceDetector = new SentenceDetectorME(new SentenceModel(new FileInputStream(new File("en-sent.bin"))));
		
		 // creates a StanfordCoreNLP object, with POS tagging, lemmatization, NER, parsing, and coreference resolution 
	    Properties props = new Properties();
//	    props.put("annotators", "tokenize, ssplit, pos, lemma, ner, parse, dcoref");
	    
	    props.put("annotators", "tokenize, ssplit, pos");
	    	    
	    StanfordCoreNLP pipeline = new StanfordCoreNLP(props);
		
		String fileCombinationsName = "/home/pjbarrio/Dataset/SGML-ACE/COMBINATIONS";
		
		loadFileForCombinations(new File(fileCombinationsName));
		
		for (int dataset = 0; dataset < Datasets.length; dataset++) {
			
			String[] RelationTags = getRelationTags(Datasets[dataset]);
			
			String extensionFile = getExtensionFile(Datasets[dataset]);
			
			int[] splits = getSplits(Datasets[dataset]);
			
			
			
			for (int spl = 0; spl < splits.length; spl++) {
					
				String rootDirectory = getDirectory(Datasets[dataset]);
				
				String directoryName = rootDirectory + spl+"/taggedtestingDirectory/";
				
				String outputFile = rootDirectory + spl+"/" + Datasets[dataset] + ".reverb.test";
				
				String frequentFile = "old_freq_rel.txt.gz";
		
				String newFrequentFile = rootDirectory+spl+"/"+Datasets[dataset]+"_acc_freq_rel.test.txt.gz";
		
				String newFrequentFileAccumulated = rootDirectory+spl+"/"+Datasets[dataset]+"_freq_rel.test.txt.gz";
				
				System.setOut(new PrintStream(new File(outputFile)));
				
//				System.setErr(new PrintStream(new File(outputFile + ".error")));
				
				File directory = new File(directoryName);
				
				File[] files = directory.listFiles();
		
		//		Arrays.sort(files);
		
				midAccumTextCount = new Hashtable<String, Integer>();
				
				midAccumTexts = new ArrayList<String>();
						
				midTextCount = new Hashtable<String, Integer>();
				
				midTexts = new ArrayList<String>();
				
				loadFrequentRelations(frequentFile);
				
				for (File file : files) {
					
					if (!file.getName().endsWith(extensionFile))
						continue;
		
					System.err.println(file.getName());
					
					SgmlDocument sgmlDocument = new SgmlDocument(new FileReader(file));
					
					Hashtable<String, List<SgmlElement>> sgmlElements = getSgmlElements(sgmlDocument,allArgument.get(Datasets[dataset]).toArray(new String[0]));
					
					Hashtable<String, List<SgmlElement>> sgmlRelationElements = getSgmlElements(sgmlDocument, RelationTags);
					
					List<Segment> sentences = getRelevantSentences(sgmlDocument,argumentsCombination.get(Datasets[dataset]),sgmlElements,allArgument.get(Datasets[dataset]).toArray(new String[0])); 
		
					for (Segment segment : sentences) {
						
						List<SgmledSegment> combinations = generateCombinations(segment,argumentsCombination.get(Datasets[dataset]),sgmlElements,allArgument.get(Datasets[dataset]).toArray(new String[0]));
						
						Hashtable<String, List<SgmlElement>> filteredRelations = filter(sgmlRelationElements, segment, RelationTags);
						
						for (SgmledSegment sgmledSegment : combinations) {
							
							// create an empty Annotation just with the given text
						    Annotation sentence = new Annotation(sgmledSegment.getText());
						    
						    // run all Annotators on this text
						    pipeline.annotate(sentence);
							
							SgmlElement first_element = sgmledSegment.getRelevantSgmlElement(1);
							
							SgmlElement second_element = sgmledSegment.getRelevantSgmlElement(2);
							
							if (!rightOrder(first_element,second_element)){
								
								SgmlElement auxiliarElement = first_element;
								first_element = second_element;
								second_element = auxiliarElement;
								
							}
							
							if (!acceptable(first_element,second_element))
								continue;
							
							List<CoreLabel> tokens = sentence.get(TokensAnnotation.class);
														
							Integer[] firstText = generateArray(first_element,tokens, sgmledSegment.getOffset());
							
							Integer[] secondText = generateArray(second_element,tokens, sgmledSegment.getOffset());
							
							if (firstText[firstText.length-1] >= (secondText[0] - 1)) //no words in the middle.
								continue;
							
							String[] words = new String[tokens.size()];
							
							String[] pos = new String[tokens.size()];
							
							int index = 0;
							
					    	for (CoreLabel token: tokens){
					    		
					    		String word = token.get(TextAnnotation.class);
					    		
					    		words[index] = word;
					    		
					    		index++;
					    		
					    		System.out.print(word + " ");
					    		
					    	}
							
					    	System.out.println();
							
					    	index = 0;
					    	
							for (CoreLabel token: tokens){
								
								String post = token.get(PartOfSpeechAnnotation.class);
								
								pos[index] = post;
								
								index++;
								
								System.out.print(post + " ");
								
							}
							
							String[] chunks = DefaultObjects.getDefaultChunker().chunk(words, pos);
							
							ChunkedSentence chunkedSentence = new ChunkedSentence(words, pos, chunks);
							
							System.out.println();

							System.out.println(chunkedSentence.getNpChunkTagsAsString());
							
							System.out.println(getText(firstText,tokens));
							
							System.out.println(firstText[0] + " " + firstText.length);
							
							String midText = getMiddleText(firstText,secondText,tokens);
							
							System.out.println(midText);
							
							updateMidTextCount(midText);
							
							int mid = secondText[0]-(firstText[firstText.length-1] + 1);
							
							int start = firstText[firstText.length-1] + 1;
							
							System.out.println(start + " " + mid);
							
							System.out.println(getText(secondText,tokens));
							
							System.out.println(secondText[0] + " " + secondText.length);
						
							System.out.println(getClass(sgmledSegment, filteredRelations, RelationTags[0],RelationTags[1]));
						}
						
						
					}
							
				}
				
				Collections.sort(midAccumTexts, new Comparator<String>(){
		
					@Override
					public int compare(String mT1, String mT2) {
						
						return midAccumTextCount.get(mT2).compareTo(midAccumTextCount.get(mT1));
						
					}
					
				});
				
				BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new GZIPOutputStream(new FileOutputStream(new File(newFrequentFileAccumulated)))));
				
				for (String midText : midAccumTexts) {
					
					bw.write(midAccumTextCount.get(midText) + "\t" + midText + "\n");
					
				}
				
				bw.close();
				
				Collections.sort(midTexts, new Comparator<String>(){
		
					@Override
					public int compare(String mT1, String mT2) {
						
						return midTextCount.get(mT2).compareTo(midTextCount.get(mT1));
						
					}
					
				});
				
				bw = new BufferedWriter(new OutputStreamWriter(new GZIPOutputStream(new FileOutputStream(new File(newFrequentFile)))));
				
				for (String midText : midTexts) {
					
					bw.write(midTextCount.get(midText) + "\t" + midText + "\n");
					
				}
				
				bw.close();
			}	
			}

	}

	private static String getMiddleText(Integer[] firstText,
			Integer[] secondText, List<CoreLabel> tokens) {
		
		
		String ret = "";
		
		int begin = firstText[firstText.length-1] + 1;
		
		int end = secondText[0];
		
		for (int i = begin; i < end; i++) {
		
			ret = ret + " " + tokens.get(i).get(TextAnnotation.class);
			
		}
		
		return ret.substring(1);
		
	}

	private static Integer[] generateArray(SgmlElement element,
			List<CoreLabel> tokens, int offset) {
		
		ArrayList<Integer> ret = new ArrayList<Integer>();
		
		int elementBegin = element.getStart() - offset;
		
		int elementEnd = element.getEnd() - offset;
		
		int index = 0;
		
		for (CoreLabel token : tokens) {

			int begin = token.beginPosition();
			
			int end = token.endPosition();

//			System.err.println(elementBegin + " - " + elementEnd + " - " + begin + " - " + end + " - " + token.get(TextAnnotation.class));
			
			if (end <= elementBegin || begin >= elementEnd){
				
				if (begin >= elementEnd){
					break;
				}
				
				index++;
				continue;
				
			}
			
	        ret.add(new Integer(index));        
			
			index++;
		}

		
		
		return ret.toArray(new Integer[0]);
		
	}

	private static boolean rightOrder(SgmlElement first_element,
			SgmlElement second_element) {
		
		return first_element.getStart() < second_element.getStart() && first_element.getEnd() < second_element.getEnd();
		
	}

	private static boolean acceptable(SgmlElement first_element,
			SgmlElement second_element) {
		
		return first_element.getEnd() <= second_element.getStart();
		
	}

	private static String getExtensionFile(String relation) {
		
		if (relation.equals(NATURAL_DISASTER)){
			return ".txt";
		}
	
		return ".sgm";
		
	}

	public static String getDirectory(String relation) {
		
		if (relation.equals(NATURAL_DISASTER)){
			return "/home/pjbarrio/Dataset/NaturalDisaster/RelatedData/reverbData/";
		}
	
		return "/home/pjbarrio/Dataset/SGML-ACE-COREF/" + relation + "/";
		
		
	}

	private static void loadFileForCombinations(File file) throws IOException {
		
		BufferedReader br = new BufferedReader(new FileReader(file));
		
		String line = br.readLine();
		
		while (line!=null){
			
			System.out.println(line);
			
			String[] split = line.split(" - ");
			
			String relation = split[0];
			
			String first = split[1];
			
			String second = split[2];
			
			Hashtable<String, Set<String>> args = argumentsCombination.get(relation);
			
			if (args == null){
				
				args = new Hashtable<String, Set<String>>();
				
			}
			
			Set<String> firstArg = args.get(first);
			
			if (firstArg == null){
				
				firstArg = new HashSet<String>();
				
			}
			
			firstArg.add(second);			
			
			args.put(first,firstArg);
			
			argumentsCombination.put(relation, args);
			
			Set<String> argsAll = allArgument.get(relation);
			
			if (argsAll == null){
				
				argsAll = new HashSet<String>();
			}
			
			argsAll.add(first);
			
			argsAll.add(second);
			
			allArgument.put(relation,argsAll);
			
			line = br.readLine();
		
		}
		
		br.close();
		
	}

	public static int[] getSplits(String relation) {
		
		if (relation.equals(NATURAL_DISASTER)){
			int[] ret = {0,1,2,3,4,5,6,7,8,9};
			return ret;
		}
	
		int[] ret = {0};
		
		return ret;
		
	}

	private static String[] getRelationTags(String relation) {
		
		if (relation.equals(NATURAL_DISASTER)){
			String[] ret = {"atLocation","locationOf"};
			return ret;
		}
	
		String[] ret = {relation + "First",relation + "Second"};
		
		return ret;
	}

	private static void loadFrequentRelations(String file) throws IOException {
		
		BufferedReader br = new BufferedReader(new InputStreamReader(
				new GZIPInputStream(ReVerbRelationDictionaryFilter.class.getClassLoader().getResourceAsStream(file))));
		
		String line = br.readLine();
		
		while (line != null){
			
			String[] split = line.split("\t");
			
			midAccumTexts.add(split[1]);
			
			midAccumTextCount.put(split[1], Integer.valueOf(split[0]));
		
			line = br.readLine();
			
		}
		
		br.close();
		
	}

	private static void updateMidTextCount(String midText) {
		
		Integer freqAccum = midAccumTextCount.get(midText);
		
		if (freqAccum == null){
			midAccumTextCount.put(midText, 1);
			midAccumTexts.add(midText);
						
		}else{
			midAccumTextCount.put(midText, freqAccum+1);
			
		}

		Integer freq = midTextCount.get(midText);
		
		if (freq == null){
			midTextCount.put(midText, 1);
			midTexts.add(midText);
			
		}else{
			midTextCount.put(midText, freq+1);
		}

		
	}


	private static String getText(Integer[] textIndex, List<CoreLabel> tokens) {
		
		String ret = "";
		
		for (Integer index : textIndex) {
			
			ret = ret + " " + tokens.get(index).get(TextAnnotation.class);
			
		}
		
		return ret.substring(1);
		
	}

	private static int getClass(SgmledSegment sgmledSegment,
			Hashtable<String, List<SgmlElement>> filteredRelations, String firstTag, String secondTag) {
		// returns 1 if the entities are related. 0 otherwise.
		
		SgmlElement relevantElement1 = sgmledSegment.getRelevantSgmlElement(1);
		
		SgmlElement relevantElement2 = sgmledSegment.getRelevantSgmlElement(2);
				
		List<SgmlElement> relevantRelationFirst = new ArrayList<SgmlElement>();
		
		List<SgmlElement> relevantRelationSecond = new ArrayList<SgmlElement>();
		
		List<SgmlElement> relations = filteredRelations.get(firstTag);
		
		for (SgmlElement relationSgmlElement : relations) {
			
			if (matches(relationSgmlElement,relevantElement1)){
				relevantRelationFirst.add(relationSgmlElement);
			}
			
		}

		
		relations = filteredRelations.get(secondTag);
		
		for (SgmlElement relationSgmlElement : relations) {
			
			if (matches(relationSgmlElement,relevantElement2)){
				relevantRelationSecond.add(relationSgmlElement);
			}
			
		}
		
		
		if (relevantRelationFirst.size() > 0 && relevantRelationSecond.size()>0){
			
			if (sameRelation(relevantRelationFirst,relevantRelationSecond)){
				return 1;
			}
			
		}
		
		return 0;
	}

	private static boolean sameRelation(List<SgmlElement> relevantRelationFirst, List<SgmlElement> relevantRelationSecond) {
		
		for (SgmlElement sgmlElement : relevantRelationFirst) {
			
			String attribute = sgmlElement.getAttribute("id");
			
			for (SgmlElement sgmlElement2 : relevantRelationSecond) {
				
				String attribute2 = sgmlElement.getAttribute("id");
				
				if (attribute.equals(attribute2)){
					return true;
				}
				
			}
			
		}
		
		return false;
		
	}

	private static boolean matches(SgmlElement relationSgmlElement,
			SgmlElement sgmlElement) {
		
		return (relationSgmlElement.getStart() >= sgmlElement.getStart() && relationSgmlElement.getEnd() <= sgmlElement.getEnd()) 
				|| (sgmlElement.getStart()>=relationSgmlElement.getStart() && sgmlElement.getEnd() <= relationSgmlElement.getEnd());
		
	}

	private static List<SgmledSegment> generateCombinations(Segment segment, Hashtable<String, Set<String>> combination, Hashtable<String, List<SgmlElement>> sgmlElements, String[] tags) {
		
		List<SgmledSegment> ret = new ArrayList<SgmledSegment>();
		
		Hashtable<String, List<SgmlElement>> containedSgmlElements = filter(sgmlElements,segment,tags);
	
		List<SgmlElement> sorted = sort(containedSgmlElements);

		Set<String> keyset = combination.keySet();
		
		for (String firstTag : keyset) {
			
			List<SgmlElement> tag_0 = containedSgmlElements.get(firstTag);
			
			Set<String> seconds = combination.get(firstTag);
			
			for (String second : seconds) {
				
				List<SgmlElement> tag_1 = containedSgmlElements.get(second);
				
				for (SgmlElement sgmlElement_0 : tag_0) {
					
					for (SgmlElement sgmlElement_1 : tag_1) {
						
						SgmledSegment aux = new SgmledSegment(segment.getText(), segment.getOffset(), sorted);
						
						aux.addSgmlElement(sgmlElement_0,1);
						
						aux.addSgmlElement(sgmlElement_1,2);
						
						ret.add(aux);
						
					}
					
				}
				
			}
		
		}
				
		
		return ret;
		
	}

	private static List<SgmlElement> sort(
			Hashtable<String, List<SgmlElement>> containedSgmlElements) {
		
		List<SgmlElement> totalSgmlElements = new ArrayList<SgmlElement>();
		
		for(Enumeration<String> e = containedSgmlElements.keys();e.hasMoreElements();){
			totalSgmlElements.addAll(containedSgmlElements.get(e.nextElement()));
		}

		Collections.sort(totalSgmlElements, new Comparator<SgmlElement>() {

			@Override
			public int compare(SgmlElement arg0, SgmlElement arg1) {
				
				return Integer.valueOf(arg0.getStart()).compareTo(arg1.getStart());
				
			}
		});
		
		return totalSgmlElements;
	}

	private static Hashtable<String, List<SgmlElement>> filter(
			Hashtable<String, List<SgmlElement>> sgmlElements, Segment segment,
			String[] tags) {
		
		Hashtable<String, List<SgmlElement>> ret = new Hashtable<String, List<SgmlElement>>();
		
		for (int i = 0; i < tags.length; i++) {
			
			List<SgmlElement> elements = sgmlElements.get(tags[i]);
			
			ArrayList<SgmlElement> aux = new ArrayList<SgmlElement>();
			
			for (SgmlElement sgmlElement : elements) {
				
				if (contains(segment, sgmlElement)){
					aux.add(sgmlElement);
				}
				
			}
			
			ret.put(tags[i], aux);
			
		}
		
		return ret;
	}

	private static List<Segment> getRelevantSentences(SgmlDocument sgmlDocument, Hashtable<String, Set<String>> combination, Hashtable<String, List<SgmlElement>> sgmlElements, String[] Tags) throws FileNotFoundException, IOException {
		
		String plainText = sgmlDocument.getSignalText();
		
		List<Segment> segments = getSegments(plainText);
		
		List<Segment> sentences = generateSentences(segments);
		
		List<Segment> fixedSentences = fixSentences(sentences,sgmlElements,Tags);
		
		List<Segment> filteredSenteces = filterSentences(sgmlElements,fixedSentences,combination);
		
		return filteredSenteces;
	}

	private static List<Segment> fixSentences(List<Segment> segments,
			Hashtable<String, List<SgmlElement>> sgmlElements, String[] tags) {

		List<Segment> ret = new ArrayList<Segment>();

		for (int j = 0; j < segments.size(); j++) {
			
			Segment seg = segments.get(j);
			
			boolean changed = false;
			
			for (int i = 0; i < tags.length && !changed; i++) {
				
				List<SgmlElement> se = sgmlElements.get(tags[i]);
				
				for (SgmlElement sgmlElement : se) {
					
					if (contains(seg, sgmlElement) && !containsItAll(seg, sgmlElement)){
						
						String link = "";
						
						int initSec = segments.get(j+1).getOffset();
						int endPrev = seg.getOffset() + seg.getText().length();
						
						int add = initSec - endPrev;
						
						for (int k = 0; k < add; k++) {
						
							link = link + " ";
						
						}
						
						segments.set(j+1, new Segment(seg.getText() + link + segments.get(j+1).getText(), seg.getOffset()));
						
						changed = true;
						
						break;
						
					}
					
				}
				
			}
			
			if (!changed)
				ret.add(seg);
			
		}
		
		return ret;
		
	}

	private static List<Segment> filterSentences(
			Hashtable<String, List<SgmlElement>> sgmlElements,
			List<Segment> sentences, Hashtable<String, Set<String>> combination) {
		// return sentences that have both tags.
		
		ArrayList<Segment> ret = new ArrayList<Segment>();
		
		Set<String> keyset = combination.keySet();

		for (Segment sentence : sentences) {
					
			boolean isIn = false;
			
			for (String first : keyset) {
				
				Set<String> seconds = combination.get(first);
				
				for (String second : seconds) {
										
					if (contains(sentence,sgmlElements.get(first)) && contains(sentence,sgmlElements.get(second))){
						isIn = true;
					}
					
					
				}
				
			}
		
			if (isIn){
				ret.add(sentence);
			}
			
		}
	
		return ret;
	}

	private static boolean contains(Segment sentence, List<SgmlElement> list) {
		
		for (SgmlElement sgmlElement : list) {
			
			if (contains(sentence,sgmlElement)){
				return true;
			}
						
		}
		
		return false;
	}

	private static boolean containsItAll(Segment sentence, SgmlElement sgmlElement) {

		int start = sentence.getOffset();
		int end = start + sentence.getText().length();

		if (sgmlElement.getStart() >= start && sgmlElement.getEnd() <= end){
			
			return true;
			
		}
		
		return false;
		
	}
	
	private static boolean contains(Segment sentence, SgmlElement sgmlElement) {
		int start = sentence.getOffset();
		int end = start + sentence.getText().length();

		if (sgmlElement.getStart() >= start && sgmlElement.getStart() <= end){
			
			return true;
			
		}
		
		return false;
		
	}

	private static Hashtable<String, List<SgmlElement>> getSgmlElements(
			SgmlDocument sgmlDocument, String[] tags) {
		
		Hashtable<String, List<SgmlElement>> ret = new Hashtable<String, List<SgmlElement>>();
		
		for (int i = 0; i < tags.length; i++) {
			
			ArrayList<SgmlElement> aux = new ArrayList<SgmlElement>();
			
			for (Iterator iterator = sgmlDocument.iterator(); iterator.hasNext();) {
				
				SgmlElement sgmlElement = (SgmlElement) iterator.next();
				
				if (sgmlElement.getGid().equals(tags[i])){
					
					aux.add(sgmlElement);
					
				}
				
			}
			
			ret.put(tags[i],aux);
			
		}
		
		return ret;
		
	}

	private static List<Segment> generateSentences(List<Segment> contentParagraph) {
		
		ArrayList<Segment> ret = new ArrayList<Segment>();
		
		for (Segment segment : contentParagraph){
			
			ret.addAll(getSentences(segment));
			
		}

		return ret;
		
	}

	private static Collection<? extends Segment> getSentences(Segment segment) {
		
		ArrayList<Segment> ret = new ArrayList<Segment>();
		
		String sentences[] = sentenceDetector.sentDetect(segment.getText());
			
		Span[] spans = sentenceDetector.sentPosDetect(segment.getText());
		
		for (int i = 0; i < sentences.length; i++) {
			
			ret.add(new Segment(sentences[i],segment.getOffset() + spans[i].getStart()));
			
		}

		return ret;
		
	}

	
	private static List<Segment> getSegments(String text) throws IOException {
		
		BufferedReader br = new BufferedReader(new StringReader(text));
		
		ArrayList<Segment> ret = new ArrayList<Segment>();
		
	    String acum = "";
	    
	    String line = br.readLine();
	     
	    int offset = 0;
	    
	    int newOffset = 0;
	    
	    while (line != null){
	    	
	    	if (line.trim().equals("")){
	    		
	    		if (!acum.trim().equals("")){
	    				    			
	    			Segment s = new Segment(acum,offset);
	    			
	    			ret.add(s);
	    		
	    		}
	    		
	    		acum = "";
	    	
	    	} else {
	    		
	    		if (acum.trim().equals("")){  //First
	    			
	    			acum = line;
	    			
	    			offset = newOffset;
	    			
	    		} else {
		    		acum = acum + " " + line;
	    		}
	    		
	    	}
	    	
	    	newOffset += line.length() + 1;
	    	
	    	line = br.readLine();
	    
	    }
	    
	    if (!acum.trim().equals("")){
	    	
	    	Segment s = new Segment(acum,offset);
	    	
	    	ret.add(s);
	    	
	    }
	    
	    br.close();
	    
	    return ret;
		
	}

	
}
